package cn.le.crawler;

import cn.hutool.http.HttpUtil;
import cn.le.utils.OkHttpUtil;
import cn.le.utils.ProxyHttpUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import okhttp3.Request;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

/**
 * @Description
 * @Author le
 * @Date 2020/4/14 10:43
 * @Version 0.1
 **/
@Component
public  class AbstractCrawler {

    protected static final ThreadPoolExecutor executor = new ThreadPoolExecutor(24,24, 10, TimeUnit.SECONDS,
            new LinkedBlockingQueue<>(),new ThreadPoolExecutor.CallerRunsPolicy());

    protected static Document getDocument(String url,int page) throws IOException {
        return getDocument(url+page);
    }

    protected static Document getDocument(String url) throws IOException {
        String content = OkHttpUtil.get(url);
        return parseDocument(content);
    }

    protected static Document parseDocument(String html){
        Document document = Jsoup.parse(html);
        return document;
    }

    protected static JSONObject getJsonResult(String url){
        String jsonStr = null;
        try {
            jsonStr = OkHttpUtil.get(url);
        } catch (IOException e) {
            e.printStackTrace();
        }
        JSONObject jsonObject = JSONObject.parseObject(jsonStr);
        return jsonObject;
    }

    protected static JSONObject getProxyJsonResult(String url){
        String jsonStr = ProxyHttpUtil.get(url);
        JSONObject jsonObject = JSONObject.parseObject(jsonStr);
        return jsonObject;
    }



    protected static JSONObject getJsonResult(Request request) throws IOException {
        String result = OkHttpUtil.get(request);
        return JSONObject.parseObject(result);
    }

    protected static String getResult(Request request) throws IOException {
        return OkHttpUtil.get(request);
    }

}
